# -*- coding: utf-8 -*-
# from __future__ import absolute_import
__author__ = 'wwj'
import sys
# reload(sys)
# sys.setdefaultencoding('utf-8')
from celery import Celery
import config
import sys
import os
from celery.utils.log import get_task_logger
import pymongo
from datetime import *
# app = Celery('dispatch',include=['tasks.xvideo'])
app = Celery('dispatch')
app.config_from_object(config)
import hashlib
import requests
from subprocess import call
import scrapy
import re
import redis
logger = get_task_logger(__name__)
pool = redis.ConnectionPool(host='localhost', port=6379)
redisconn = redis.Redis(connection_pool=pool)


def getdb(self,sql):
    cursor = self.conn.cursor()
    cursor.execute(sql)
    tmp = cursor.fetchallDict()
    return tmp

@app.task(bind=True,queue="city_clean",max_retries=1,ignore_result=True)
def prepare(self,item):
    if not self.redisconn.exists("stores"):
            tmp = self.getdb("select distinct(url),id from store")
            for x in tmp:
                self.redisconn.hset("stores",x["url"],x["id"])



    str(item)(item)

def clean(self,y):
        data = self.data
        y = defaultdict(lambda:"",y)
        url = y["storesUrl"].replace("https://www.cityheaven.net","")
        url = re.sub("girlid.*/","",url)
        if not url:
            url = re.sub("girlid.*/","",y["url"])

        tmp = self.redisconn.hget("stores",url)
        if tmp:
            data['sid'] = str(tmp)
        else:
            self.myException("no sid")
        data["age"] = re.search("\d+",y["age"]).group() if re.search("\d+",y["age"]) else ""
        data["name"] = y["name"]
        bwh = re.findall("\d+",y["bwh"])
        if len(bwh) > 3:
            data["height"] = bwh[0] or ""
            data["bust"] = bwh[1] or ""
            data["waist"] = bwh[2] or ""
            data["hipline"] = bwh[3] or ""
        else:
            data["height"] = data["bust"] = data["waist"] = data["hipline"] = ''
        if re.search(r"\((.+)\)",y["bwh"]):
            data["cup"] = re.search(r"\((.+)\)",y["bwh"]).groups()[0]
        data["blood"] = y["blood"]
        tmp = y.get("image_urls")
        tmp = map(lambda url: re.sub("\?cache.*", "", url), tmp)
        data["imgs"] = ",".join(tmp)
        data["constellation"] = y["constellation"]
        data["thumb"] = tmp[0] if tmp else ""
        data["url"] = y["url"].replace("https://www.cityheaven.net","")
        data["urlHash"] = str(crc32(data["url"])).replace("-","")
        return buildsql(data)

def buildsql(data):
    data_values = "(" + "%s," * (len(data.keys())) + ")"
    data_values = data_values.replace(',)', ')')
    keys = str(tuple(keys)).replace("'",'')
    values = [data]
    for value in values:
        dd = []
        sql = "insert ignore into %s %s values %s" % (self.mysqlTable,keys,data_values)
        # sql = "replace into %s %s values %s" % (self.mysqlTable,keys,data_values)
        # sql = "INSERT INTO auto %s(%s) VALUES (%s) ON DUPLICATE KEY UPDATE urlHash=VALUES(urlHash)"
        for v in value:
            dd.append([MySQLdb.escape_string(str(y).strip().replace("%","")) for yy,y in v.iteritems()])
        try:
            pp(dd)
            self.conn.ping(True)
            self.cursor = self.conn.cursor()
            self.cursor.executemany(sql,dd)
            self.conn.commit()
            del self.datas[:self.exesize]
        except Exception as ex:
            # import ipdb;ipdb.set_trace()
            self.conn.ping(True)
            print ex
